This code is Script 2 for Kitchel et al. TITLE manuscript.

SESSION INFO TO DO

library(tidyverse)
library(data.table)
data.table 1.14.4 using 1 threads (see ?getDTthreads).  Latest news: r-datatable.com
**********
This installation of data.table has not detected OpenMP support. It should still work but in single-threaded mode.
This is a Mac. Please read https://mac.r-project.org/openmp/. Please engage with Apple and ask them for support. Check r-datatable.com for updates, and our Mac instructions here: https://github.com/Rdatatable/data.table/wiki/Installation. After several years of many reports of installation problems on Mac, it's time to gingerly point out that there have been no similar problems on Windows or Linux.
**********

Attaching package: ‘data.table’

The following object is masked from ‘package:raster’:

    shift

The following objects are masked from ‘package:dplyr’:

    between, first, last

The following object is masked from ‘package:purrr’:

    transpose
library(here)
here() starts at /Users/zoekitchel/Documents/grad_school/Rutgers/Repositories/trawl_spatial_turnover_git
library(sp)
library(raster)
library(rgeos)
library(rgbif)
library(viridis)
library(gridExtra)
library(rasterVis)
library(concaveman)
library(sf)
library(viridis)
set.seed(1)

Load manually reduced fishglob database from prepare_fishglob_dataset.Rmd

FishGlob.10year.spp_manualclean <- readRDS(here::here("data","cleaned","FishGlob.10year.spp_manualclean.rds"))

####Edit function to create grid which will allow us to maintain similar spatial footprint over time From here

We will use cell size of 7,774.2 km^2, as that will match grid cell size of 8 in dggridr. We can’t use the package dggridr unfortunately because it doesn’t work for this version of R (and others have had this issue too). https://github.com/r-barnes/dggridR For Norway we will use cell size of 23,322.2 km^2 because the sites are further away from each other.

Make sampling locations into spatial points

#delete if NA for longitude or latitude
FishGlob.10year.spp_manualclean <- FishGlob.10year.spp_manualclean[complete.cases(FishGlob.10year.spp_manualclean[,.(longitude, latitude)])] 

Match lat/lon sampling points to hexagonal cells, so that we can see how many cells to keep to maintain a lot of observation points

FishGlob.cells <- data.table()

#two potential cell sizes
  #set cell area (depends on whether or not it's Norway)
  cell_area_all  <- 7774.2 #km2 (8 from dggrdr)
  cell_area_norway <- 23322.2 #km2 (7 from dggrdr; if you want to use different resolution, not doing as of now)
  
all_survey_units <- unique(FishGlob.10year.spp_manualclean[,survey_unit])
  

for(i in 1:length(all_survey_units)){
  FishGlob.10year.spp_manualclean_subset <- FishGlob.10year.spp_manualclean[survey_unit == all_survey_units[i],]
  
  #unique lat lon combos
  FishGlob.10year.spp_manualclean_subset_unique <- unique(FishGlob.10year.spp_manualclean_subset[,.(longitude,latitude,haul_id,year)])
  
  #coordinates to Spatial Points Object
  if(max(FishGlob.10year.spp_manualclean_subset_unique[,longitude]) - min(FishGlob.10year.spp_manualclean_subset_unique[,longitude]) > 359){ #if survey region crosses dateline, use st_shift_longitude()
    sp <- FishGlob.10year.spp_manualclean_subset_unique %>%
          st_as_sf(coords = c("longitude","latitude"), crs = 4326) %>%
             st_shift_longitude()
  }else{
    sp <- FishGlob.10year.spp_manualclean_subset_unique %>%
          st_as_sf(coords = c("longitude","latitude"), crs = 4326)
  }
  
  sp.t <- as(sp, "Spatial")
  
  proj4string(sp.t) <- CRS("+proj=longlat")
  
proj <-  ifelse(max(FishGlob.10year.spp_manualclean_subset_unique[,longitude]) - min(FishGlob.10year.spp_manualclean_subset_unique[,longitude]) > 359, #if survey region crosses dateline, use +lon_0=-140 instead of +lon_0=0
         "+proj=robin +lon_0=-140 +x_0=0 +y_0=0 +ellps=WGS84 +datum=WGS84 +units=km +no_defs",
         "+proj=robin +lon_0=0 +x_0=0 +y_0=0 +ellps=WGS84 +datum=WGS84 +units=km +no_defs")
  
  
  sp.p <- spTransform(sp.t, CRS(proj)) #note km^2 units
  
  #use Concaveman to convert points to polygon
  polygon  <- concaveman(sp, 2, 3)

  polygon_spapol <- as(polygon, "Spatial") #convert simple polygon feature to spatial polygon
  
  proj4string(polygon_spapol) <- CRS("+proj=longlat")
  
  polygon_spapol.p <- spTransform(polygon_spapol, CRS(proj)) #note km^2 units
  
  #create grid 
  #set cell_area based on whether or not it's the Norway survey
  cell_area_km <- ifelse(all_survey_units[i] %in% c("Nor-BTS-1","Nor-BTS-3"), cell_area_norway, cell_area_all) #note this is in kilometers (if you want to use different cell resolution for Norway)
  
  #calculate cell_diameter of hexagons from cell_areas
  cell_diameter_km <- sqrt(2 * cell_area_km / sqrt(3)) # in meters
  

  ext <- as(extent(polygon_spapol.p)
            + 2*cell_diameter_km #add a buffer to make sure all observations are assigned a cell
             , "SpatialPolygons")
 # plot(ext)
 # plot(sp.p, add = T, pch = ".")
  
  projection(ext) <- projection(polygon_spapol.p) #match projection
  
  # generate array of hexagon centers
  g <- spsample(ext, type = "hexagonal", cellsize = cell_diameter_km, offset = c(0.5, 0.5))
  
    # convert center points to hexagons
  g <- HexPoints2SpatialPolygons(g, dx = cell_diameter_km)
  
 plot(g)
 plot(sp.p, add = T, pch = ".")
 title(paste0(all_survey_units[i]))

  #link lat lon to cell#
    #where do they overlap
    sp.p$cell_ID <- over(sp.p,g) #over(x=location of queries, y = layer from which geometries are queried)
    
    #link lat long to cell #s
    FishGlob.10year.spp_manualclean_subset_unique[,cell_ID := sp.p$cell_ID][,cell_year_count := .N, .(cell_ID, year)]
    
    #link back to subsetted database of observations
    FishGlob.subset.cells <- FishGlob.10year.spp_manualclean_subset[FishGlob.10year.spp_manualclean_subset_unique, on = c("longitude", "latitude","year","haul_id")]
    
    FishGlob.cells <- rbind(FishGlob.cells, FishGlob.subset.cells)

    
#make sure all projections match for binding of polygons
polygon_spapol.forbind <- spTransform(polygon_spapol.p,
                                      CRS=CRS( "+proj=robin +lon_0=0 +x_0=0 +y_0=0 +ellps=WGS84 +datum=WGS84 +units=km +no_defs"))

      polygon_spapol.forbind$survey_unit <- all_survey_units[i]
      
    #bind polygons into spdf
    if(i ==1){
      all_survey_units_polygon <- polygon_spapol.forbind #if first, just polygon to start
    }else{
      all_survey_units_polygon <- rbind(all_survey_units_polygon, polygon_spapol.forbind)  #if not first, bind new polygon to first polygon
    }
    
       
  
}
Warning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among othersWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.htmlWarning: CRS object has comment, which is lost in output; in tests, see
https://cran.r-project.org/web/packages/sp/vignettes/CRS_warnings.html

Warning: PROJ support is provided by the sf and terra packages among others

  
writeOGR(all_survey_units_polygon, dsn = here::here("output/shapefiles"))
Error in writeOGR(all_survey_units_polygon, dsn = here::here("output/shapefiles")) : 
  could not find function "writeOGR"

Merge all shapefiles together in order to send to Sea Around Us team to obtain fishing specific fishing data.


Standardize observations by # of hauls per cell and # cells sampled per year

Remove any years that sample less than 70% of cells ever sampled Remove any cells that are sampled in less than 70% of years


#account of data loss by standardization
data_loss <- data.table(survey_unit=all_survey_units,
                        year_threshold=as.numeric(NA),
                        cell_threshold=as.numeric(NA),
                        percent_years_excluded=as.numeric(NA),
                        percent_hauls_excluded_by_year=as.numeric(NA),
                        percent_obs_excluded_by_year=as.numeric(NA),
                        percent_cells_excluded=as.numeric(NA),
                        percent_hauls_excluded_total=as.numeric(NA),
                        percent_obs_excluded_total=as.numeric(NA))

FishGlob.wellsampledyearscells_complete <- data.table()

for (i in 1:length(all_survey_units)) {
      
      #subset to region
      FishGlob.cells.subset <- FishGlob.cells[survey_unit == all_survey_units[i],]
      
      #unique year, #cells  sampled
      year_cells_sampled <- unique(FishGlob.cells.subset[,.(year,cell_ID)])
      year_cells_sampled <- year_cells_sampled[,yearly_cell_count := .N,year]
      year_cells_sampled <- unique(year_cells_sampled[,.(year,yearly_cell_count)])
      
      #eliminate years with less than 70% of cells sampled
      year_benchmark <- 0.70
      benchmark_value <- year_benchmark*max(year_cells_sampled[,yearly_cell_count])
      
      #only keep years where over 70% of cells are sampled
      year_cells_sampled[,benchmark := yearly_cell_count >= benchmark_value]
      
      years_deleted <- unique(year_cells_sampled[benchmark == F,year]) #which years are left out?
      
      years_kept <-unique(year_cells_sampled[benchmark ==T,year]) #which years to keep
      
      years_deleted_percent <- round(length(years_deleted)/length(unique(year_cells_sampled[,year]))*100,1)
      

       #print the years that are left out
      print(ifelse(length(years_deleted) == 0, paste0(all_survey_units[i], " Years left out = 0"), paste0(" ",all_survey_units[i], " Years left out = ", years_deleted, collapse = ",")))
     
     print(paste0(years_deleted_percent, "% of Years Excluded"))
      
      #reduce to years that are well sampled
      FishGlob.cells.subset.wellsampledyears <- FishGlob.cells.subset[year %in% years_kept,]
      
      #how many observations does this remove?
      percent_obs_removed_year <- round((nrow(FishGlob.cells.subset)-nrow(FishGlob.cells.subset.wellsampledyears))/nrow(FishGlob.cells.subset)*100,2)
      
      percent_hauls_removed_year <- round((length(unique(FishGlob.cells.subset[,haul_id]))-length(unique(FishGlob.cells.subset.wellsampledyears[,haul_id])))/length(unique(FishGlob.cells.subset[,haul_id]))*100,2)
      
      #identify any cells that are not sampled in 70% of years
      FishGlob.cells.subset.wellsampledyears[,year_cell_count := length(unique(haul_id)),.(year,cell_ID)] # unique haul ids per cell per year
      
      cell_by_year <- unique(FishGlob.cells.subset.wellsampledyears[, .(cell_ID,year)])
      
      cell_by_year[,years_per_cell := .N,cell_ID]
      
      #cell ids to remove and keep
      #in any year, which cells are sampled in less than 70% of years
      #we'll make benchmark 70% just for now
      cell_benchmark <- 0.70
      benchmark_value_year_count <- cell_benchmark*max(cell_by_year[,years_per_cell])
      
      cell_id_remove <- unique(cell_by_year[years_per_cell<benchmark_value_year_count,cell_ID])
      
      cells_deleted_percent <- round(length(cell_id_remove)/length(unique(FishGlob.cells.subset.wellsampledyears[,cell_ID]))*100,1)
      
      #reduce to cells that are well sampled
      FishGlob.cells.subset.wellsampledyearscells <- FishGlob.cells.subset.wellsampledyears[!(cell_ID %in% cell_id_remove),]
      
      #What percent of hauls does this remove?
      hauls_removed_yearcell <- round((length(unique(FishGlob.cells.subset[,haul_id]))-length(unique(FishGlob.cells.subset.wellsampledyearscells[,haul_id])))/length(unique(FishGlob.cells.subset[,haul_id]))*100,1) 
      
      #What percent of observations does this remove?
      obs_removed_yearcell <- round((nrow(FishGlob.cells.subset)-nrow(FishGlob.cells.subset.wellsampledyearscells))/nrow(FishGlob.cells.subset)*100,1)
      
      cell_id_remove.string <- paste(cell_id_remove, collapse = ", ")
      obs_removed.string <- paste(obs_removed_yearcell, collapse = ", ")
      
      #build data table from this reduced output
      FishGlob.wellsampledyearscells_complete <- rbind(FishGlob.wellsampledyearscells_complete, FishGlob.cells.subset.wellsampledyearscells)
      
      #fill out table with statistics of dropped observations
      data_loss[i, "year_threshold"] = year_benchmark
      data_loss[i, "cell_threshold"] = cell_benchmark
      data_loss[i, "percent_years_excluded"] = years_deleted_percent
      data_loss[i, "percent_hauls_excluded_by_year"] = percent_hauls_removed_year
      data_loss[i, "percent_obs_excluded_by_year"] = percent_obs_removed_year
      data_loss[i, "percent_cells_excluded"] = cells_deleted_percent
      data_loss[i, "percent_hauls_excluded_total"] = hauls_removed_yearcell
      data_loss[i, "percent_obs_excluded_total"] = obs_removed_yearcell
      
        #print portion of cells that are left out
      print(ifelse(length(cell_id_remove) == 0, paste0(all_survey_units[i], " Cells left out = 0"), paste0(all_survey_units[i], " Cells left out = ", cell_id_remove.string, ", ",cells_deleted_percent, "% Cells Excluded, ",hauls_removed_yearcell,"% Hauls Removed, ", obs_removed_yearcell, "% Observations Removed")))
      
      }
[1] " AI Years left out = 1991"
[1] "7.1% of Years Excluded"
[1] "AI Cells left out = 27, 53, 134, 88, 10% Cells Excluded, 5.4% Hauls Removed, 5.1% Observations Removed"
[1] " CHL Years left out = 2010"
[1] "5.9% of Years Excluded"
[1] "CHL Cells left out = 46, 17, 10, 5, 4, 22, 26, 30.4% Cells Excluded, 9.7% Hauls Removed, 11% Observations Removed"
[1] "BITS-1 Years left out = 0"
[1] "0% of Years Excluded"
[1] "BITS-1 Cells left out = 50, 22, 65, 53, 78, 19, 80, 69, 20, 20.9% Cells Excluded, 3.3% Hauls Removed, 2.9% Observations Removed"
[1] "BITS-4 Years left out = 0"
[1] "0% of Years Excluded"
[1] "BITS-4 Cells left out = 6, 36, 24, 26, 41, 22, 78, 5, 20, 66, 79, 24.4% Cells Excluded, 4.3% Hauls Removed, 3.9% Observations Removed"
[1] "EVHOE Years left out = 0"
[1] "0% of Years Excluded"
[1] "EVHOE Cells left out = 42, 49, 72, 119, 67, 118, 100, 117, 123, 112, 90, 23.4% Cells Excluded, 3.1% Hauls Removed, 3.8% Observations Removed"
[1] " FR-CGFS Years left out = 2020"
[1] "4.3% of Years Excluded"
[1] "FR-CGFS Cells left out = 3, 9.1% Cells Excluded, 2.9% Hauls Removed, 4.3% Observations Removed"
[1] " IE-IGFS Years left out = 2011"
[1] "6.2% of Years Excluded"
[1] "IE-IGFS Cells left out = 50, 34, 64, 59, 26, 2, 3, 19.4% Cells Excluded, 9.6% Hauls Removed, 9.5% Observations Removed"
[1] " NIGFS-1 Years left out = 2006, NIGFS-1 Years left out = 2007"
[1] "13.3% of Years Excluded"
[1] "NIGFS-1 Cells left out = 0"
[1] " NIGFS-4 Years left out = 2006, NIGFS-4 Years left out = 2007"
[1] "14.3% of Years Excluded"
[1] "NIGFS-4 Cells left out = 0"
[1] "NS-IBTS-1 Years left out = 0"
[1] "0% of Years Excluded"
[1] "NS-IBTS-1 Cells left out = 112, 187, 127, 82, 168, 188, 194, 180, 105, 200, 37, 52, 139, 203, 20, 36, 21, 35, 19, 4, 219, 34, 167, 253, 87, 22.5% Cells Excluded, 3.2% Hauls Removed, 4% Observations Removed"
[1] "NS-IBTS-3 Years left out = 0"
[1] "0% of Years Excluded"
[1] "NS-IBTS-3 Cells left out = 73, 126, 21, 151, 89, 141, 163, 104, 37, 188, 108, 172, 96, 156, 157, 221, 81, 17.3% Cells Excluded, 2.7% Hauls Removed, 3.1% Observations Removed"
[1] " PT-IBTS Years left out = 2018"
[1] "7.1% of Years Excluded"
[1] "PT-IBTS Cells left out = 0"
[1] " ROCKALL Years left out = 1999, ROCKALL Years left out = 2001, ROCKALL Years left out = 2002, ROCKALL Years left out = 2003, ROCKALL Years left out = 2005, ROCKALL Years left out = 2006, ROCKALL Years left out = 2007, ROCKALL Years left out = 2008, ROCKALL Years left out = 2009"
[1] "47.4% of Years Excluded"
[1] "ROCKALL Cells left out = 16, 6, 2, 37.5% Cells Excluded, 48% Hauls Removed, 44.7% Observations Removed"
[1] " SWC-IBTS-1 Years left out = 1985, SWC-IBTS-1 Years left out = 1986, SWC-IBTS-1 Years left out = 1990, SWC-IBTS-1 Years left out = 1992, SWC-IBTS-1 Years left out = 1993, SWC-IBTS-1 Years left out = 1994, SWC-IBTS-1 Years left out = 2007, SWC-IBTS-1 Years left out = 2009, SWC-IBTS-1 Years left out = 2010"
[1] "25.7% of Years Excluded"
[1] "SWC-IBTS-1 Cells left out = 19, 22, 21, 57, 74, 63, 36, 18, 10, 53, 39, 75, 73, 11, 46, 85, 84, 67, 76, 58, 48, 15, 24, 14, 5, 6, 61.9% Cells Excluded, 39.9% Hauls Removed, 39% Observations Removed"
[1] " SWC-IBTS-4 Years left out = 2013, SWC-IBTS-4 Years left out = 2017, SWC-IBTS-4 Years left out = 2018, SWC-IBTS-4 Years left out = 2019, SWC-IBTS-4 Years left out = 2020"
[1] "20.8% of Years Excluded"
[1] "SWC-IBTS-4 Cells left out = 148, 147, 134, 111, 90, 89, 76, 64, 39, 27, 38, 48, 36, 15, 4, 16, 17, 149, 66, 55, 54, 43, 56, 68, 67, 51% Cells Excluded, 28.9% Hauls Removed, 27.5% Observations Removed"
[1] " DFO-NF Years left out = 2014"
[1] "4% of Years Excluded"
[1] "DFO-NF Cells left out = 49, 66, 60, 59, 70, 79, 89, 96, 106, 88, 93, 84, 149, 164, 165, 153, 145, 166, 46, 126, 21, 5, 22% Cells Excluded, 6.1% Hauls Removed, 6.5% Observations Removed"
[1] "EBS Years left out = 0"
[1] "0% of Years Excluded"
[1] "EBS Cells left out = 163, 1.1% Cells Excluded, 0% Hauls Removed, 0% Observations Removed"
[1] " FALK Years left out = 2006, FALK Years left out = 2007, FALK Years left out = 2010, FALK Years left out = 2020, FALK Years left out = 2021"
[1] "38.5% of Years Excluded"
[1] "FALK Cells left out = 51, 41, 50, 49, 59, 60, 61, 70, 71, 69, 58, 68, 78, 77, 79, 88, 80, 81, 90, 87, 89, 98, 99, 108, 38, 47, 25, 26, 18, 27, 72, 37, 35, 15, 16, 7, 69.2% Cells Excluded, 54% Hauls Removed, 50.8% Observations Removed"
[1] " GMEX-Fall Years left out = 2003, GMEX-Fall Years left out = 1996, GMEX-Fall Years left out = 1991, GMEX-Fall Years left out = 1994, GMEX-Fall Years left out = 2002, GMEX-Fall Years left out = 2004, GMEX-Fall Years left out = 1989, GMEX-Fall Years left out = 1993, GMEX-Fall Years left out = 1998, GMEX-Fall Years left out = 1988, GMEX-Fall Years left out = 1990, GMEX-Fall Years left out = 1992, GMEX-Fall Years left out = 1999, GMEX-Fall Years left out = 1997, GMEX-Fall Years left out = 2001, GMEX-Fall Years left out = 2000, GMEX-Fall Years left out = 1995, GMEX-Fall Years left out = 2006, GMEX-Fall Years left out = 2005, GMEX-Fall Years left out = 2007, GMEX-Fall Years left out = 2011"
[1] "63.6% of Years Excluded"
[1] "GMEX-Fall Cells left out = 123, 109, 47, 30, 31, 49, 66, 82, 115, 131, 147, 112, 52, 91, 33, 26.3% Cells Excluded, 66.5% Hauls Removed, 64.7% Observations Removed"
[1] "GMEX-Summer Years left out = 0"
[1] "0% of Years Excluded"
[1] "GMEX-Summer Cells left out = 43, 26, 69, 57, 12.9% Cells Excluded, 1.4% Hauls Removed, 1.7% Observations Removed"
[1] " GOA Years left out = 2001"
[1] "6.2% of Years Excluded"
[1] "GOA Cells left out = 40, 41, 76, 77, 83, 78, 114, 1, 2, 37, 38, 39, 152, 312, 229, 282, 305, 387, 248, 319, 354, 379, 388, 191, 213, 267, 382, 42, 28% Cells Excluded, 8.7% Hauls Removed, 7.6% Observations Removed"
[1] " GRL-DE Years left out = 2016"
[1] "4.2% of Years Excluded"
[1] "GRL-DE Cells left out = 193, 9, 174, 195, 173, 153, 152, 196, 175, 120, 32, 31, 167, 166, 31.8% Cells Excluded, 8.2% Hauls Removed, 6.5% Observations Removed"
[1] "GSL-N Years left out = 0"
[1] "0% of Years Excluded"
[1] "GSL-N Cells left out = 55, 116, 71, 45, 10.8% Cells Excluded, 0.7% Hauls Removed, 0.9% Observations Removed"
[1] "GSL-S Years left out = 0"
[1] "0% of Years Excluded"
[1] "GSL-S Cells left out = 0"
[1] "ICE-GFS Years left out = 0"
[1] "0% of Years Excluded"
[1] "ICE-GFS Cells left out = 36, 2% Cells Excluded, 0.1% Hauls Removed, 0.1% Observations Removed"
[1] "MEDITS Years left out = 0"
[1] "0% of Years Excluded"
[1] "MEDITS Cells left out = 78, 79, 80, 81, 239, 318, 280, 277, 478, 476, 516, 326, 363, 244, 97, 57, 58, 59, 98, 95, 595, 404, 408, 175, 182, 221, 183, 220, 219, 218, 256, 144, 180, 257, 181, 294, 184, 142, 143, 105, 222, 185, 147, 148, 146, 107, 145, 224, 69, 68, 106, 67, 76, 37, 75, 77, 38, 37.3% Cells Excluded, 9.9% Hauls Removed, 10.6% Observations Removed"
[1] " NAM Years left out = 2003, NAM Years left out = 2007, NAM Years left out = 2009, NAM Years left out = 2018"
[1] "20% of Years Excluded"
[1] "NAM Cells left out = 12, 36, 87, 88, 94, 118, 32, 82, 26, 105, 22, 452, 451, 95, 35% Cells Excluded, 19.7% Hauls Removed, 19.8% Observations Removed"
[1] " NEUS-Fall Years left out = 1963, NEUS-Fall Years left out = 1964, NEUS-Fall Years left out = 1965, NEUS-Fall Years left out = 1966, NEUS-Fall Years left out = 1967, NEUS-Fall Years left out = 1981, NEUS-Fall Years left out = 2017"
[1] "14.6% of Years Excluded"
[1] "NEUS-Fall Cells left out = 176, 238, 257, 276, 277, 275, 258, 312, 178, 199, 296, 295, 40, 58, 39, 20, 38, 1, 2, 21, 41, 78, 232, 177, 311, 115, 290, 37% Cells Excluded, 10.6% Hauls Removed, 10.1% Observations Removed"
[1] " NEUS-Spring Years left out = 1977, NEUS-Spring Years left out = 2020"
[1] "4.4% of Years Excluded"
[1] "NEUS-Spring Cells left out = 178, 276, 258, 257, 177, 277, 275, 238, 295, 296, 219, 312, 41, 40, 21, 20, 2, 1, 38, 39, 58, 78, 232, 115, 251, 34.2% Cells Excluded, 3.5% Hauls Removed, 3.4% Observations Removed"
[1] " Nor-BTS-1 Years left out = 2002, Nor-BTS-1 Years left out = 2003, Nor-BTS-1 Years left out = 2004, Nor-BTS-1 Years left out = 2006, Nor-BTS-1 Years left out = 2007, Nor-BTS-1 Years left out = 2008, Nor-BTS-1 Years left out = 2009, Nor-BTS-1 Years left out = 2012"
[1] "47.1% of Years Excluded"
[1] "Nor-BTS-1 Cells left out = 21, 114, 115, 116, 136, 206, 111, 112, 113, 133, 23, 24, 101, 190, 83, 181, 182, 183, 205, 105, 109, 156, 198, 199, 180, 200, 201, 202, 203, 204, 207, 208, 171, 209, 210, 172, 191, 173, 174, 175, 217, 47.1% Cells Excluded, 51.8% Hauls Removed, 54.2% Observations Removed"
[1] "Nor-BTS-3 Years left out = 0"
[1] "0% of Years Excluded"
[1] "Nor-BTS-3 Cells left out = 99, 125, 277, 201, 284, 122, 154, 177, 245, 285, 288, 289, 267, 77, 76, 54, 223, 168, 261, 310, 178, 78, 268, 224, 290, 246, 291, 269, 247, 270, 29, 7, 30, 31, 239, 53, 32, 137, 114, 136, 309, 276, 260, 55, 39.3% Cells Excluded, 10.5% Hauls Removed, 11% Observations Removed"
[1] "NZ-WCSI Years left out = 0"
[1] "0% of Years Excluded"
[1] "NZ-WCSI Cells left out = 39, 49, 1, 20% Cells Excluded, 2.1% Hauls Removed, 1.4% Observations Removed"
[1] "NZ-ECSI Years left out = 0"
[1] "0% of Years Excluded"
[1] "NZ-ECSI Cells left out = 32, 10% Cells Excluded, 1.2% Hauls Removed, 1.3% Observations Removed"
[1] "NZ-CHAT Years left out = 0"
[1] "0% of Years Excluded"
[1] "NZ-CHAT Cells left out = 46, 25, 24, 47, 23, 3, 1, 14, 10, 40, 2, 45, 48, 44, 4, 40.5% Cells Excluded, 8.4% Hauls Removed, 9% Observations Removed"
[1] "NZ-SUBA Years left out = 0"
[1] "0% of Years Excluded"
[1] "NZ-SUBA Cells left out = 98, 84, 45, 30, 43, 29, 16, 3, 2, 41, 39, 66, 56, 118, 132, 122, 109, 82, 94, 92, 87, 59, 71, 32, 4, 28, 52, 53, 58, 19, 93, 74, 99, 96, 97, 123, 57.1% Cells Excluded, 29.8% Hauls Removed, 27.9% Observations Removed"
[1] "DFO-QCS Years left out = 0"
[1] "0% of Years Excluded"
[1] "DFO-QCS Cells left out = 14, 9.1% Cells Excluded, 0.1% Hauls Removed, 0.1% Observations Removed"
[1] " S-GEORG Years left out = 2005"
[1] "6.2% of Years Excluded"
[1] "S-GEORG Cells left out = 13, 17, 8, 15, 32, 29.4% Cells Excluded, 8.4% Hauls Removed, 8.8% Observations Removed"
[1] " SCS-SPRING Years left out = 1987, SCS-SPRING Years left out = 1988, SCS-SPRING Years left out = 1989, SCS-SPRING Years left out = 1990, SCS-SPRING Years left out = 1992, SCS-SPRING Years left out = 1996, SCS-SPRING Years left out = 1997, SCS-SPRING Years left out = 1998, SCS-SPRING Years left out = 1999, SCS-SPRING Years left out = 2000, SCS-SPRING Years left out = 2001, SCS-SPRING Years left out = 2002, SCS-SPRING Years left out = 2003, SCS-SPRING Years left out = 2005, SCS-SPRING Years left out = 2010, SCS-SPRING Years left out = 1993, SCS-SPRING Years left out = 2007, SCS-SPRING Years left out = 2018, SCS-SPRING Years left out = 1991, SCS-SPRING Years left out = 1995, SCS-SPRING Years left out = 2009, SCS-SPRING Years left out = 2017, SCS-SPRING Years left out = 2008, SCS-SPRING Years left out = 1986, SCS-SPRING Years left out = 2006, SCS-SPRING Years left out = 2012, SCS-SPRING Years left out = 2020, SCS-SPRING Years left out = 2013, SCS-SPRING Years left out = 2004, SCS-SPRING Years left out = 2011"
[1] "78.9% of Years Excluded"
[1] "SCS-SPRING Cells left out = 112, 95, 113, 128, 129, 130, 115, 116, 48, 143, 109, 33, 34, 47, 107, 50, 62, 32, 31, 16, 18, 19, 17, 45, 2, 144, 50% Cells Excluded, 79% Hauls Removed, 78.2% Observations Removed"
[1] " SCS-SUMMER Years left out = 2018"
[1] "2.9% of Years Excluded"
[1] "SCS-SUMMER Cells left out = 63, 50, 77, 43, 14, 102, 13, 62, 72, 69, 57, 89, 2, 74, 29, 30.6% Cells Excluded, 5.2% Hauls Removed, 5.7% Observations Removed"
[1] "SEUS-spring Years left out = 0"
[1] "0% of Years Excluded"
[1] "SEUS-spring Cells left out = 85, 2, 13.3% Cells Excluded, 2% Hauls Removed, 2.1% Observations Removed"
[1] "SEUS-summer Years left out = 0"
[1] "0% of Years Excluded"
[1] "SEUS-summer Cells left out = 2, 37, 12.5% Cells Excluded, 1.1% Hauls Removed, 1.1% Observations Removed"
[1] " SEUS-fall Years left out = 2018, SEUS-fall Years left out = 2019"
[1] "6.7% of Years Excluded"
[1] "SEUS-fall Cells left out = 85, 56, 2, 18.8% Cells Excluded, 6.9% Hauls Removed, 7.4% Observations Removed"
[1] "WCANN Years left out = 0"
[1] "0% of Years Excluded"
[1] "WCANN Cells left out = 157, 10, 13, 3, 4, 10.9% Cells Excluded, 0.5% Hauls Removed, 0.4% Observations Removed"
[1] "ZAF-IND Years left out = 0"
[1] "0% of Years Excluded"
[1] "ZAF-IND Cells left out = 10, 18, 36, 33, 21, 2, 28.6% Cells Excluded, 4.1% Hauls Removed, 3.7% Observations Removed"
[1] "ZAF-ATL Years left out = 0"
[1] "0% of Years Excluded"
[1] "ZAF-ATL Cells left out = 13, 25, 67, 31, 6, 23, 23.1% Cells Excluded, 2.8% Hauls Removed, 3.1% Observations Removed"
     


#now, check again to see if any are less than 10 years
FishGlob.wellsampledyearscells_complete[,years_sampled_update := length(unique(year)),.(survey_unit)]
FishGlob.wellsampledyearscells_complete.10year <- FishGlob.wellsampledyearscells_complete[years_sampled_update >= 10,]

#saveRDS(FishGlob_cleaned.10year, here::here("output","region_season_cleaned_data","FishGlob_cleaned.10year.rds"))

#FishGlob_cleaned.10year <- readRDS(here::here("output","region_season_cleaned_data","FishGlob_cleaned.10year.rds"))
LS0tCnRpdGxlOiAiQXBwbHkgVGVtcG9yYWwgYW5kIFNwYXRpYWwgU3RhbmRhcmRpemF0aW9uIgpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sKLS0tCgpUaGlzIGNvZGUgaXMgU2NyaXB0IDIgZm9yIEtpdGNoZWwgZXQgYWwuIFRJVExFIG1hbnVzY3JpcHQuCgotIFRoaXMgcHJvamVjdCBpcyBhIGNvbGxhYm9yYXRpdmUgZWZmb3J0IHRvIGRlc2NyaWJlIGNoYW5nZXMgaW4gdGF4b25vbWljIGNvbXBvc2l0aW9uICBvZiBmaXNoIGNvbW11bml0aWVzIGFyb3VuZCB0aGUgd29ybGQtLWFzIHNhbXBsZWQgYnkgYm90dG9tIHRyYXdsIHN1cnZleXMuCgotIENvZGUgYnkgWm/DqyBKLiBLaXRjaGVsCgpTRVNTSU9OIElORk8gVE8gRE8KCgpgYGB7ciBzZXR1cH0KbGlicmFyeSh0aWR5dmVyc2UpCmxpYnJhcnkoZGF0YS50YWJsZSkKbGlicmFyeShoZXJlKQpsaWJyYXJ5KHNwKQpsaWJyYXJ5KHJhc3RlcikKbGlicmFyeShyZ2VvcykKbGlicmFyeShyZ2JpZikKbGlicmFyeSh2aXJpZGlzKQpsaWJyYXJ5KGdyaWRFeHRyYSkKbGlicmFyeShyYXN0ZXJWaXMpCmxpYnJhcnkoY29uY2F2ZW1hbikKbGlicmFyeShzZikKbGlicmFyeSh2aXJpZGlzKQpzZXQuc2VlZCgxKQpgYGAKCkxvYWQgbWFudWFsbHkgcmVkdWNlZCBmaXNoZ2xvYiBkYXRhYmFzZSBmcm9tIHByZXBhcmVfZmlzaGdsb2JfZGF0YXNldC5SbWQKYGBge3J9CkZpc2hHbG9iLjEweWVhci5zcHBfbWFudWFsY2xlYW4gPC0gcmVhZFJEUyhoZXJlOjpoZXJlKCJkYXRhIiwiY2xlYW5lZCIsIkZpc2hHbG9iLjEweWVhci5zcHBfbWFudWFsY2xlYW4ucmRzIikpCmBgYAoKCiMjIyNFZGl0IGZ1bmN0aW9uIHRvIGNyZWF0ZSBncmlkIHdoaWNoIHdpbGwgYWxsb3cgdXMgdG8gbWFpbnRhaW4gc2ltaWxhciBzcGF0aWFsIGZvb3RwcmludCBvdmVyIHRpbWUKRnJvbSBbaGVyZV0oaHR0cHM6Ly9zdHJpbWFzLmNvbS9wb3N0L2hleGFnb25hbC1ncmlkcy8pCgpXZSB3aWxsIHVzZSBjZWxsIHNpemUgb2YgNyw3NzQuMiBrbV4yLCBhcyB0aGF0IHdpbGwgbWF0Y2ggZ3JpZCBjZWxsIHNpemUgb2YgOCBpbiBkZ2dyaWRyLiBXZSBjYW4ndCB1c2UgdGhlIHBhY2thZ2UgZGdncmlkciB1bmZvcnR1bmF0ZWx5IGJlY2F1c2UgaXQgZG9lc24ndCB3b3JrIGZvciB0aGlzIHZlcnNpb24gb2YgUiAoYW5kIG90aGVycyBoYXZlIGhhZCB0aGlzIGlzc3VlIHRvbykuIGh0dHBzOi8vZ2l0aHViLmNvbS9yLWJhcm5lcy9kZ2dyaWRSCkZvciBOb3J3YXkgd2Ugd2lsbCB1c2UgY2VsbCBzaXplIG9mIDIzLDMyMi4yIGttXjIgYmVjYXVzZSB0aGUgc2l0ZXMgYXJlIGZ1cnRoZXIgYXdheSBmcm9tIGVhY2ggb3RoZXIuCgpNYWtlIHNhbXBsaW5nIGxvY2F0aW9ucyBpbnRvIHNwYXRpYWwgcG9pbnRzCmBgYHtyIHNhbXBsaW5nIGxvY2F0aW9ucyB0byBzcH0KI2RlbGV0ZSBpZiBOQSBmb3IgbG9uZ2l0dWRlIG9yIGxhdGl0dWRlCkZpc2hHbG9iLjEweWVhci5zcHBfbWFudWFsY2xlYW4gPC0gRmlzaEdsb2IuMTB5ZWFyLnNwcF9tYW51YWxjbGVhbltjb21wbGV0ZS5jYXNlcyhGaXNoR2xvYi4xMHllYXIuc3BwX21hbnVhbGNsZWFuWywuKGxvbmdpdHVkZSwgbGF0aXR1ZGUpXSldIAoKYGBgCgpNYXRjaCBsYXQvbG9uIHNhbXBsaW5nIHBvaW50cyB0byBoZXhhZ29uYWwgY2VsbHMsIHNvIHRoYXQgd2UgY2FuIHNlZSBob3cgbWFueSBjZWxscyB0byBrZWVwIHRvIG1haW50YWluIGEgbG90IG9mIG9ic2VydmF0aW9uIHBvaW50cwoKYGBge3IgbWF0Y2ggbGF0IGxvbiB0byBoZXggY2VsbHN9CkZpc2hHbG9iLmNlbGxzIDwtIGRhdGEudGFibGUoKQoKI3R3byBwb3RlbnRpYWwgY2VsbCBzaXplcwogICNzZXQgY2VsbCBhcmVhIChkZXBlbmRzIG9uIHdoZXRoZXIgb3Igbm90IGl0J3MgTm9yd2F5KQogIGNlbGxfYXJlYV9hbGwgIDwtIDc3NzQuMiAja20yICg4IGZyb20gZGdncmRyKQogIGNlbGxfYXJlYV9ub3J3YXkgPC0gMjMzMjIuMiAja20yICg3IGZyb20gZGdncmRyOyBpZiB5b3Ugd2FudCB0byB1c2UgZGlmZmVyZW50IHJlc29sdXRpb24sIG5vdCBkb2luZyBhcyBvZiBub3cpCiAgCmFsbF9zdXJ2ZXlfdW5pdHMgPC0gdW5pcXVlKEZpc2hHbG9iLjEweWVhci5zcHBfbWFudWFsY2xlYW5bLHN1cnZleV91bml0XSkKICAKCmZvcihpIGluIDE6bGVuZ3RoKGFsbF9zdXJ2ZXlfdW5pdHMpKXsKICBGaXNoR2xvYi4xMHllYXIuc3BwX21hbnVhbGNsZWFuX3N1YnNldCA8LSBGaXNoR2xvYi4xMHllYXIuc3BwX21hbnVhbGNsZWFuW3N1cnZleV91bml0ID09IGFsbF9zdXJ2ZXlfdW5pdHNbaV0sXQogIAogICN1bmlxdWUgbGF0IGxvbiBjb21ib3MKICBGaXNoR2xvYi4xMHllYXIuc3BwX21hbnVhbGNsZWFuX3N1YnNldF91bmlxdWUgPC0gdW5pcXVlKEZpc2hHbG9iLjEweWVhci5zcHBfbWFudWFsY2xlYW5fc3Vic2V0WywuKGxvbmdpdHVkZSxsYXRpdHVkZSxoYXVsX2lkLHllYXIpXSkKICAKICAjY29vcmRpbmF0ZXMgdG8gU3BhdGlhbCBQb2ludHMgT2JqZWN0CiAgaWYobWF4KEZpc2hHbG9iLjEweWVhci5zcHBfbWFudWFsY2xlYW5fc3Vic2V0X3VuaXF1ZVssbG9uZ2l0dWRlXSkgLSBtaW4oRmlzaEdsb2IuMTB5ZWFyLnNwcF9tYW51YWxjbGVhbl9zdWJzZXRfdW5pcXVlWyxsb25naXR1ZGVdKSA+IDM1OSl7ICNpZiBzdXJ2ZXkgcmVnaW9uIGNyb3NzZXMgZGF0ZWxpbmUsIHVzZSBzdF9zaGlmdF9sb25naXR1ZGUoKQogICAgc3AgPC0gRmlzaEdsb2IuMTB5ZWFyLnNwcF9tYW51YWxjbGVhbl9zdWJzZXRfdW5pcXVlICU+JQogICAgICAgICAgc3RfYXNfc2YoY29vcmRzID0gYygibG9uZ2l0dWRlIiwibGF0aXR1ZGUiKSwgY3JzID0gNDMyNikgJT4lCiAgICAgICAgICAgICBzdF9zaGlmdF9sb25naXR1ZGUoKQogIH1lbHNlewogICAgc3AgPC0gRmlzaEdsb2IuMTB5ZWFyLnNwcF9tYW51YWxjbGVhbl9zdWJzZXRfdW5pcXVlICU+JQogICAgICAgICAgc3RfYXNfc2YoY29vcmRzID0gYygibG9uZ2l0dWRlIiwibGF0aXR1ZGUiKSwgY3JzID0gNDMyNikKICB9CiAgCiAgc3AudCA8LSBhcyhzcCwgIlNwYXRpYWwiKQogIAogIHByb2o0c3RyaW5nKHNwLnQpIDwtIENSUygiK3Byb2o9bG9uZ2xhdCIpCiAgCnByb2ogPC0gIGlmZWxzZShtYXgoRmlzaEdsb2IuMTB5ZWFyLnNwcF9tYW51YWxjbGVhbl9zdWJzZXRfdW5pcXVlWyxsb25naXR1ZGVdKSAtIG1pbihGaXNoR2xvYi4xMHllYXIuc3BwX21hbnVhbGNsZWFuX3N1YnNldF91bmlxdWVbLGxvbmdpdHVkZV0pID4gMzU5LCAjaWYgc3VydmV5IHJlZ2lvbiBjcm9zc2VzIGRhdGVsaW5lLCB1c2UgK2xvbl8wPS0xNDAgaW5zdGVhZCBvZiArbG9uXzA9MAogICAgICAgICAiK3Byb2o9cm9iaW4gK2xvbl8wPS0xNDAgK3hfMD0wICt5XzA9MCArZWxscHM9V0dTODQgK2RhdHVtPVdHUzg0ICt1bml0cz1rbSArbm9fZGVmcyIsCiAgICAgICAgICIrcHJvaj1yb2JpbiArbG9uXzA9MCAreF8wPTAgK3lfMD0wICtlbGxwcz1XR1M4NCArZGF0dW09V0dTODQgK3VuaXRzPWttICtub19kZWZzIikKICAKICAKICBzcC5wIDwtIHNwVHJhbnNmb3JtKHNwLnQsIENSUyhwcm9qKSkgI25vdGUga21eMiB1bml0cwogIAogICN1c2UgQ29uY2F2ZW1hbiB0byBjb252ZXJ0IHBvaW50cyB0byBwb2x5Z29uCiAgcG9seWdvbiAgPC0gY29uY2F2ZW1hbihzcCwgMiwgMykKCiAgcG9seWdvbl9zcGFwb2wgPC0gYXMocG9seWdvbiwgIlNwYXRpYWwiKSAjY29udmVydCBzaW1wbGUgcG9seWdvbiBmZWF0dXJlIHRvIHNwYXRpYWwgcG9seWdvbgogIAogIHByb2o0c3RyaW5nKHBvbHlnb25fc3BhcG9sKSA8LSBDUlMoIitwcm9qPWxvbmdsYXQiKQogIAogIHBvbHlnb25fc3BhcG9sLnAgPC0gc3BUcmFuc2Zvcm0ocG9seWdvbl9zcGFwb2wsIENSUyhwcm9qKSkgI25vdGUga21eMiB1bml0cwogIAogICNjcmVhdGUgZ3JpZCAKICAjc2V0IGNlbGxfYXJlYSBiYXNlZCBvbiB3aGV0aGVyIG9yIG5vdCBpdCdzIHRoZSBOb3J3YXkgc3VydmV5CiAgY2VsbF9hcmVhX2ttIDwtIGlmZWxzZShhbGxfc3VydmV5X3VuaXRzW2ldICVpbiUgYygiTm9yLUJUUy0xIiwiTm9yLUJUUy0zIiksIGNlbGxfYXJlYV9ub3J3YXksIGNlbGxfYXJlYV9hbGwpICNub3RlIHRoaXMgaXMgaW4ga2lsb21ldGVycyAoaWYgeW91IHdhbnQgdG8gdXNlIGRpZmZlcmVudCBjZWxsIHJlc29sdXRpb24gZm9yIE5vcndheSkKICAKICAjY2FsY3VsYXRlIGNlbGxfZGlhbWV0ZXIgb2YgaGV4YWdvbnMgZnJvbSBjZWxsX2FyZWFzCiAgY2VsbF9kaWFtZXRlcl9rbSA8LSBzcXJ0KDIgKiBjZWxsX2FyZWFfa20gLyBzcXJ0KDMpKSAjIGluIG1ldGVycwogIAoKICBleHQgPC0gYXMoZXh0ZW50KHBvbHlnb25fc3BhcG9sLnApCiAgICAgICAgICAgICsgMipjZWxsX2RpYW1ldGVyX2ttICNhZGQgYSBidWZmZXIgdG8gbWFrZSBzdXJlIGFsbCBvYnNlcnZhdGlvbnMgYXJlIGFzc2lnbmVkIGEgY2VsbAogICAgICAgICAgICAgLCAiU3BhdGlhbFBvbHlnb25zIikKICMgcGxvdChleHQpCiAjIHBsb3Qoc3AucCwgYWRkID0gVCwgcGNoID0gIi4iKQogIAogIHByb2plY3Rpb24oZXh0KSA8LSBwcm9qZWN0aW9uKHBvbHlnb25fc3BhcG9sLnApICNtYXRjaCBwcm9qZWN0aW9uCiAgCiAgIyBnZW5lcmF0ZSBhcnJheSBvZiBoZXhhZ29uIGNlbnRlcnMKICBnIDwtIHNwc2FtcGxlKGV4dCwgdHlwZSA9ICJoZXhhZ29uYWwiLCBjZWxsc2l6ZSA9IGNlbGxfZGlhbWV0ZXJfa20sIG9mZnNldCA9IGMoMC41LCAwLjUpKQogIAogICAgIyBjb252ZXJ0IGNlbnRlciBwb2ludHMgdG8gaGV4YWdvbnMKICBnIDwtIEhleFBvaW50czJTcGF0aWFsUG9seWdvbnMoZywgZHggPSBjZWxsX2RpYW1ldGVyX2ttKQogIAogcGxvdChnKQogcGxvdChzcC5wLCBhZGQgPSBULCBwY2ggPSAiLiIpCiB0aXRsZShwYXN0ZTAoYWxsX3N1cnZleV91bml0c1tpXSkpCgogICNsaW5rIGxhdCBsb24gdG8gY2VsbCMKICAgICN3aGVyZSBkbyB0aGV5IG92ZXJsYXAKICAgIHNwLnAkY2VsbF9JRCA8LSBvdmVyKHNwLnAsZykgI292ZXIoeD1sb2NhdGlvbiBvZiBxdWVyaWVzLCB5ID0gbGF5ZXIgZnJvbSB3aGljaCBnZW9tZXRyaWVzIGFyZSBxdWVyaWVkKQogICAgCiAgICAjbGluayBsYXQgbG9uZyB0byBjZWxsICNzCiAgICBGaXNoR2xvYi4xMHllYXIuc3BwX21hbnVhbGNsZWFuX3N1YnNldF91bmlxdWVbLGNlbGxfSUQgOj0gc3AucCRjZWxsX0lEXVssY2VsbF95ZWFyX2NvdW50IDo9IC5OLCAuKGNlbGxfSUQsIHllYXIpXQogICAgCiAgICAjbGluayBiYWNrIHRvIHN1YnNldHRlZCBkYXRhYmFzZSBvZiBvYnNlcnZhdGlvbnMKICAgIEZpc2hHbG9iLnN1YnNldC5jZWxscyA8LSBGaXNoR2xvYi4xMHllYXIuc3BwX21hbnVhbGNsZWFuX3N1YnNldFtGaXNoR2xvYi4xMHllYXIuc3BwX21hbnVhbGNsZWFuX3N1YnNldF91bmlxdWUsIG9uID0gYygibG9uZ2l0dWRlIiwgImxhdGl0dWRlIiwieWVhciIsImhhdWxfaWQiKV0KICAgIAogICAgRmlzaEdsb2IuY2VsbHMgPC0gcmJpbmQoRmlzaEdsb2IuY2VsbHMsIEZpc2hHbG9iLnN1YnNldC5jZWxscykKCiAgICAKI21ha2Ugc3VyZSBhbGwgcHJvamVjdGlvbnMgbWF0Y2ggZm9yIGJpbmRpbmcgb2YgcG9seWdvbnMKcG9seWdvbl9zcGFwb2wuZm9yYmluZCA8LSBzcFRyYW5zZm9ybShwb2x5Z29uX3NwYXBvbC5wLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIENSUz1DUlMoICIrcHJvaj1yb2JpbiArbG9uXzA9MCAreF8wPTAgK3lfMD0wICtlbGxwcz1XR1M4NCArZGF0dW09V0dTODQgK3VuaXRzPWttICtub19kZWZzIikpCgogICAgICBwb2x5Z29uX3NwYXBvbC5mb3JiaW5kJHN1cnZleV91bml0IDwtIGFsbF9zdXJ2ZXlfdW5pdHNbaV0KICAgICAgCiAgICAjYmluZCBwb2x5Z29ucyBpbnRvIHNwZGYKICAgIGlmKGkgPT0xKXsKICAgICAgYWxsX3N1cnZleV91bml0c19wb2x5Z29uIDwtIHBvbHlnb25fc3BhcG9sLmZvcmJpbmQgI2lmIGZpcnN0LCBqdXN0IHBvbHlnb24gdG8gc3RhcnQKICAgIH1lbHNlewogICAgICBhbGxfc3VydmV5X3VuaXRzX3BvbHlnb24gPC0gcmJpbmQoYWxsX3N1cnZleV91bml0c19wb2x5Z29uLCBwb2x5Z29uX3NwYXBvbC5mb3JiaW5kKSAgI2lmIG5vdCBmaXJzdCwgYmluZCBuZXcgcG9seWdvbiB0byBmaXJzdCBwb2x5Z29uCiAgICB9CiAgICAKICAgICAgIAogIAp9CiAgCiN3cml0ZU9HUihhbGxfc3VydmV5X3VuaXRzX3BvbHlnb24sIGRzbiA9IGhlcmU6OmhlcmUoIm91dHB1dC9zaGFwZWZpbGVzIikpCmBgYAoKCgoKTWVyZ2UgYWxsIHNoYXBlZmlsZXMgdG9nZXRoZXIgaW4gb3JkZXIgdG8gc2VuZCB0byBTZWEgQXJvdW5kIFVzIHRlYW0gdG8gb2J0YWluIGZpc2hpbmcgc3BlY2lmaWMgZmlzaGluZyBkYXRhLgpgYGB7cn0KCmBgYAoKCl9fX19fX19fCgpTdGFuZGFyZGl6ZSBvYnNlcnZhdGlvbnMgYnkgIyBvZiBoYXVscyBwZXIgY2VsbCBhbmQgIyBjZWxscyBzYW1wbGVkIHBlciB5ZWFyCgpSZW1vdmUgYW55IHllYXJzIHRoYXQgc2FtcGxlIGxlc3MgdGhhbiA3MCUgb2YgY2VsbHMgIGV2ZXIgc2FtcGxlZApSZW1vdmUgYW55IGNlbGxzIHRoYXQgYXJlIHNhbXBsZWQgaW4gbGVzcyB0aGFuIDcwJSBvZiB5ZWFycwoKYGBge3Igc3RhbmRhcmRpemUgb2JzZXJ2YXRpb25zIGFsbCByZWdpb25zfQoKI2FjY291bnQgb2YgZGF0YSBsb3NzIGJ5IHN0YW5kYXJkaXphdGlvbgpkYXRhX2xvc3MgPC0gZGF0YS50YWJsZShzdXJ2ZXlfdW5pdD1hbGxfc3VydmV5X3VuaXRzLAogICAgICAgICAgICAgICAgICAgICAgICB5ZWFyX3RocmVzaG9sZD1hcy5udW1lcmljKE5BKSwKICAgICAgICAgICAgICAgICAgICAgICAgY2VsbF90aHJlc2hvbGQ9YXMubnVtZXJpYyhOQSksCiAgICAgICAgICAgICAgICAgICAgICAgIHBlcmNlbnRfeWVhcnNfZXhjbHVkZWQ9YXMubnVtZXJpYyhOQSksCiAgICAgICAgICAgICAgICAgICAgICAgIHBlcmNlbnRfaGF1bHNfZXhjbHVkZWRfYnlfeWVhcj1hcy5udW1lcmljKE5BKSwKICAgICAgICAgICAgICAgICAgICAgICAgcGVyY2VudF9vYnNfZXhjbHVkZWRfYnlfeWVhcj1hcy5udW1lcmljKE5BKSwKICAgICAgICAgICAgICAgICAgICAgICAgcGVyY2VudF9jZWxsc19leGNsdWRlZD1hcy5udW1lcmljKE5BKSwKICAgICAgICAgICAgICAgICAgICAgICAgcGVyY2VudF9oYXVsc19leGNsdWRlZF90b3RhbD1hcy5udW1lcmljKE5BKSwKICAgICAgICAgICAgICAgICAgICAgICAgcGVyY2VudF9vYnNfZXhjbHVkZWRfdG90YWw9YXMubnVtZXJpYyhOQSkpCgpGaXNoR2xvYi53ZWxsc2FtcGxlZHllYXJzY2VsbHNfY29tcGxldGUgPC0gZGF0YS50YWJsZSgpCgpmb3IgKGkgaW4gMTpsZW5ndGgoYWxsX3N1cnZleV91bml0cykpIHsKICAgICAgCiAgICAgICNzdWJzZXQgdG8gcmVnaW9uCiAgICAgIEZpc2hHbG9iLmNlbGxzLnN1YnNldCA8LSBGaXNoR2xvYi5jZWxsc1tzdXJ2ZXlfdW5pdCA9PSBhbGxfc3VydmV5X3VuaXRzW2ldLF0KICAgICAgCiAgICAgICN1bmlxdWUgeWVhciwgI2NlbGxzICBzYW1wbGVkCiAgICAgIHllYXJfY2VsbHNfc2FtcGxlZCA8LSB1bmlxdWUoRmlzaEdsb2IuY2VsbHMuc3Vic2V0WywuKHllYXIsY2VsbF9JRCldKQogICAgICB5ZWFyX2NlbGxzX3NhbXBsZWQgPC0geWVhcl9jZWxsc19zYW1wbGVkWyx5ZWFybHlfY2VsbF9jb3VudCA6PSAuTix5ZWFyXQogICAgICB5ZWFyX2NlbGxzX3NhbXBsZWQgPC0gdW5pcXVlKHllYXJfY2VsbHNfc2FtcGxlZFssLih5ZWFyLHllYXJseV9jZWxsX2NvdW50KV0pCiAgICAgIAogICAgICAjZWxpbWluYXRlIHllYXJzIHdpdGggbGVzcyB0aGFuIDcwJSBvZiBjZWxscyBzYW1wbGVkCiAgICAgIHllYXJfYmVuY2htYXJrIDwtIDAuNzAKICAgICAgYmVuY2htYXJrX3ZhbHVlIDwtIHllYXJfYmVuY2htYXJrKm1heCh5ZWFyX2NlbGxzX3NhbXBsZWRbLHllYXJseV9jZWxsX2NvdW50XSkKICAgICAgCiAgICAgICNvbmx5IGtlZXAgeWVhcnMgd2hlcmUgb3ZlciA3MCUgb2YgY2VsbHMgYXJlIHNhbXBsZWQKICAgICAgeWVhcl9jZWxsc19zYW1wbGVkWyxiZW5jaG1hcmsgOj0geWVhcmx5X2NlbGxfY291bnQgPj0gYmVuY2htYXJrX3ZhbHVlXQogICAgICAKICAgICAgeWVhcnNfZGVsZXRlZCA8LSB1bmlxdWUoeWVhcl9jZWxsc19zYW1wbGVkW2JlbmNobWFyayA9PSBGLHllYXJdKSAjd2hpY2ggeWVhcnMgYXJlIGxlZnQgb3V0PwogICAgICAKICAgICAgeWVhcnNfa2VwdCA8LXVuaXF1ZSh5ZWFyX2NlbGxzX3NhbXBsZWRbYmVuY2htYXJrID09VCx5ZWFyXSkgI3doaWNoIHllYXJzIHRvIGtlZXAKICAgICAgCiAgICAgIHllYXJzX2RlbGV0ZWRfcGVyY2VudCA8LSByb3VuZChsZW5ndGgoeWVhcnNfZGVsZXRlZCkvbGVuZ3RoKHVuaXF1ZSh5ZWFyX2NlbGxzX3NhbXBsZWRbLHllYXJdKSkqMTAwLDEpCiAgICAgIAoKICAgICAgICNwcmludCB0aGUgeWVhcnMgdGhhdCBhcmUgbGVmdCBvdXQKICAgICAgcHJpbnQoaWZlbHNlKGxlbmd0aCh5ZWFyc19kZWxldGVkKSA9PSAwLCBwYXN0ZTAoYWxsX3N1cnZleV91bml0c1tpXSwgIiBZZWFycyBsZWZ0IG91dCA9IDAiKSwgcGFzdGUwKCIgIixhbGxfc3VydmV5X3VuaXRzW2ldLCAiIFllYXJzIGxlZnQgb3V0ID0gIiwgeWVhcnNfZGVsZXRlZCwgY29sbGFwc2UgPSAiLCIpKSkKICAgICAKICAgICBwcmludChwYXN0ZTAoeWVhcnNfZGVsZXRlZF9wZXJjZW50LCAiJSBvZiBZZWFycyBFeGNsdWRlZCIpKQogICAgICAKICAgICAgI3JlZHVjZSB0byB5ZWFycyB0aGF0IGFyZSB3ZWxsIHNhbXBsZWQKICAgICAgRmlzaEdsb2IuY2VsbHMuc3Vic2V0LndlbGxzYW1wbGVkeWVhcnMgPC0gRmlzaEdsb2IuY2VsbHMuc3Vic2V0W3llYXIgJWluJSB5ZWFyc19rZXB0LF0KICAgICAgCiAgICAgICNob3cgbWFueSBvYnNlcnZhdGlvbnMgZG9lcyB0aGlzIHJlbW92ZT8KICAgICAgcGVyY2VudF9vYnNfcmVtb3ZlZF95ZWFyIDwtIHJvdW5kKChucm93KEZpc2hHbG9iLmNlbGxzLnN1YnNldCktbnJvdyhGaXNoR2xvYi5jZWxscy5zdWJzZXQud2VsbHNhbXBsZWR5ZWFycykpL25yb3coRmlzaEdsb2IuY2VsbHMuc3Vic2V0KSoxMDAsMikKICAgICAgCiAgICAgIHBlcmNlbnRfaGF1bHNfcmVtb3ZlZF95ZWFyIDwtIHJvdW5kKChsZW5ndGgodW5pcXVlKEZpc2hHbG9iLmNlbGxzLnN1YnNldFssaGF1bF9pZF0pKS1sZW5ndGgodW5pcXVlKEZpc2hHbG9iLmNlbGxzLnN1YnNldC53ZWxsc2FtcGxlZHllYXJzWyxoYXVsX2lkXSkpKS9sZW5ndGgodW5pcXVlKEZpc2hHbG9iLmNlbGxzLnN1YnNldFssaGF1bF9pZF0pKSoxMDAsMikKICAgICAgCiAgICAgICNpZGVudGlmeSBhbnkgY2VsbHMgdGhhdCBhcmUgbm90IHNhbXBsZWQgaW4gNzAlIG9mIHllYXJzCiAgICAgIEZpc2hHbG9iLmNlbGxzLnN1YnNldC53ZWxsc2FtcGxlZHllYXJzWyx5ZWFyX2NlbGxfY291bnQgOj0gbGVuZ3RoKHVuaXF1ZShoYXVsX2lkKSksLih5ZWFyLGNlbGxfSUQpXSAjIHVuaXF1ZSBoYXVsIGlkcyBwZXIgY2VsbCBwZXIgeWVhcgogICAgICAKICAgICAgY2VsbF9ieV95ZWFyIDwtIHVuaXF1ZShGaXNoR2xvYi5jZWxscy5zdWJzZXQud2VsbHNhbXBsZWR5ZWFyc1ssIC4oY2VsbF9JRCx5ZWFyKV0pCiAgICAgIAogICAgICBjZWxsX2J5X3llYXJbLHllYXJzX3Blcl9jZWxsIDo9IC5OLGNlbGxfSURdCiAgICAgIAogICAgICAjY2VsbCBpZHMgdG8gcmVtb3ZlIGFuZCBrZWVwCiAgICAgICNpbiBhbnkgeWVhciwgd2hpY2ggY2VsbHMgYXJlIHNhbXBsZWQgaW4gbGVzcyB0aGFuIDcwJSBvZiB5ZWFycwogICAgICAjd2UnbGwgbWFrZSBiZW5jaG1hcmsgNzAlIGp1c3QgZm9yIG5vdwogICAgICBjZWxsX2JlbmNobWFyayA8LSAwLjcwCiAgICAgIGJlbmNobWFya192YWx1ZV95ZWFyX2NvdW50IDwtIGNlbGxfYmVuY2htYXJrKm1heChjZWxsX2J5X3llYXJbLHllYXJzX3Blcl9jZWxsXSkKICAgICAgCiAgICAgIGNlbGxfaWRfcmVtb3ZlIDwtIHVuaXF1ZShjZWxsX2J5X3llYXJbeWVhcnNfcGVyX2NlbGw8YmVuY2htYXJrX3ZhbHVlX3llYXJfY291bnQsY2VsbF9JRF0pCiAgICAgIAogICAgICBjZWxsc19kZWxldGVkX3BlcmNlbnQgPC0gcm91bmQobGVuZ3RoKGNlbGxfaWRfcmVtb3ZlKS9sZW5ndGgodW5pcXVlKEZpc2hHbG9iLmNlbGxzLnN1YnNldC53ZWxsc2FtcGxlZHllYXJzWyxjZWxsX0lEXSkpKjEwMCwxKQogICAgICAKICAgICAgI3JlZHVjZSB0byBjZWxscyB0aGF0IGFyZSB3ZWxsIHNhbXBsZWQKICAgICAgRmlzaEdsb2IuY2VsbHMuc3Vic2V0LndlbGxzYW1wbGVkeWVhcnNjZWxscyA8LSBGaXNoR2xvYi5jZWxscy5zdWJzZXQud2VsbHNhbXBsZWR5ZWFyc1shKGNlbGxfSUQgJWluJSBjZWxsX2lkX3JlbW92ZSksXQogICAgICAKICAgICAgI1doYXQgcGVyY2VudCBvZiBoYXVscyBkb2VzIHRoaXMgcmVtb3ZlPwogICAgICBoYXVsc19yZW1vdmVkX3llYXJjZWxsIDwtIHJvdW5kKChsZW5ndGgodW5pcXVlKEZpc2hHbG9iLmNlbGxzLnN1YnNldFssaGF1bF9pZF0pKS1sZW5ndGgodW5pcXVlKEZpc2hHbG9iLmNlbGxzLnN1YnNldC53ZWxsc2FtcGxlZHllYXJzY2VsbHNbLGhhdWxfaWRdKSkpL2xlbmd0aCh1bmlxdWUoRmlzaEdsb2IuY2VsbHMuc3Vic2V0WyxoYXVsX2lkXSkpKjEwMCwxKSAKICAgICAgCiAgICAgICNXaGF0IHBlcmNlbnQgb2Ygb2JzZXJ2YXRpb25zIGRvZXMgdGhpcyByZW1vdmU/CiAgICAgIG9ic19yZW1vdmVkX3llYXJjZWxsIDwtIHJvdW5kKChucm93KEZpc2hHbG9iLmNlbGxzLnN1YnNldCktbnJvdyhGaXNoR2xvYi5jZWxscy5zdWJzZXQud2VsbHNhbXBsZWR5ZWFyc2NlbGxzKSkvbnJvdyhGaXNoR2xvYi5jZWxscy5zdWJzZXQpKjEwMCwxKQogICAgICAKICAgICAgY2VsbF9pZF9yZW1vdmUuc3RyaW5nIDwtIHBhc3RlKGNlbGxfaWRfcmVtb3ZlLCBjb2xsYXBzZSA9ICIsICIpCiAgICAgIG9ic19yZW1vdmVkLnN0cmluZyA8LSBwYXN0ZShvYnNfcmVtb3ZlZF95ZWFyY2VsbCwgY29sbGFwc2UgPSAiLCAiKQogICAgICAKICAgICAgI2J1aWxkIGRhdGEgdGFibGUgZnJvbSB0aGlzIHJlZHVjZWQgb3V0cHV0CiAgICAgIEZpc2hHbG9iLndlbGxzYW1wbGVkeWVhcnNjZWxsc19jb21wbGV0ZSA8LSByYmluZChGaXNoR2xvYi53ZWxsc2FtcGxlZHllYXJzY2VsbHNfY29tcGxldGUsIEZpc2hHbG9iLmNlbGxzLnN1YnNldC53ZWxsc2FtcGxlZHllYXJzY2VsbHMpCiAgICAgIAogICAgICAjZmlsbCBvdXQgdGFibGUgd2l0aCBzdGF0aXN0aWNzIG9mIGRyb3BwZWQgb2JzZXJ2YXRpb25zCiAgICAgIGRhdGFfbG9zc1tpLCAieWVhcl90aHJlc2hvbGQiXSA9IHllYXJfYmVuY2htYXJrCiAgICAgIGRhdGFfbG9zc1tpLCAiY2VsbF90aHJlc2hvbGQiXSA9IGNlbGxfYmVuY2htYXJrCiAgICAgIGRhdGFfbG9zc1tpLCAicGVyY2VudF95ZWFyc19leGNsdWRlZCJdID0geWVhcnNfZGVsZXRlZF9wZXJjZW50CiAgICAgIGRhdGFfbG9zc1tpLCAicGVyY2VudF9oYXVsc19leGNsdWRlZF9ieV95ZWFyIl0gPSBwZXJjZW50X2hhdWxzX3JlbW92ZWRfeWVhcgogICAgICBkYXRhX2xvc3NbaSwgInBlcmNlbnRfb2JzX2V4Y2x1ZGVkX2J5X3llYXIiXSA9IHBlcmNlbnRfb2JzX3JlbW92ZWRfeWVhcgogICAgICBkYXRhX2xvc3NbaSwgInBlcmNlbnRfY2VsbHNfZXhjbHVkZWQiXSA9IGNlbGxzX2RlbGV0ZWRfcGVyY2VudAogICAgICBkYXRhX2xvc3NbaSwgInBlcmNlbnRfaGF1bHNfZXhjbHVkZWRfdG90YWwiXSA9IGhhdWxzX3JlbW92ZWRfeWVhcmNlbGwKICAgICAgZGF0YV9sb3NzW2ksICJwZXJjZW50X29ic19leGNsdWRlZF90b3RhbCJdID0gb2JzX3JlbW92ZWRfeWVhcmNlbGwKICAgICAgCiAgICAgICAgI3ByaW50IHBvcnRpb24gb2YgY2VsbHMgdGhhdCBhcmUgbGVmdCBvdXQKICAgICAgcHJpbnQoaWZlbHNlKGxlbmd0aChjZWxsX2lkX3JlbW92ZSkgPT0gMCwgcGFzdGUwKGFsbF9zdXJ2ZXlfdW5pdHNbaV0sICIgQ2VsbHMgbGVmdCBvdXQgPSAwIiksIHBhc3RlMChhbGxfc3VydmV5X3VuaXRzW2ldLCAiIENlbGxzIGxlZnQgb3V0ID0gIiwgY2VsbF9pZF9yZW1vdmUuc3RyaW5nLCAiLCAiLGNlbGxzX2RlbGV0ZWRfcGVyY2VudCwgIiUgQ2VsbHMgRXhjbHVkZWQsICIsaGF1bHNfcmVtb3ZlZF95ZWFyY2VsbCwiJSBIYXVscyBSZW1vdmVkLCAiLCBvYnNfcmVtb3ZlZF95ZWFyY2VsbCwgIiUgT2JzZXJ2YXRpb25zIFJlbW92ZWQiKSkpCiAgICAgIAogICAgICB9CiAgICAgCgoKI25vdywgY2hlY2sgYWdhaW4gdG8gc2VlIGlmIGFueSBhcmUgbGVzcyB0aGFuIDEwIHllYXJzCkZpc2hHbG9iLndlbGxzYW1wbGVkeWVhcnNjZWxsc19jb21wbGV0ZVsseWVhcnNfc2FtcGxlZF91cGRhdGUgOj0gbGVuZ3RoKHVuaXF1ZSh5ZWFyKSksLihzdXJ2ZXlfdW5pdCldCkZpc2hHbG9iLndlbGxzYW1wbGVkeWVhcnNjZWxsc19jb21wbGV0ZS4xMHllYXIgPC0gRmlzaEdsb2Iud2VsbHNhbXBsZWR5ZWFyc2NlbGxzX2NvbXBsZXRlW3llYXJzX3NhbXBsZWRfdXBkYXRlID49IDEwLF0KCiNvbmx5IGtlZXAgc3VydmV5IHVuaXRzIGZvciB3aGljaCBmZXdlciB0aGFuIDIwJSBvZiBvYnNlcnZhdGlvbnMgd2VyZSBkZWxldGVkIGJ5IDcwJSB0aHJlc2hvbGRzCnN1cnZleV91bml0c19rZWVwIDwtIHVuaXF1ZShkYXRhX2xvc3NbcGVyY2VudF9vYnNfZXhjbHVkZWRfdG90YWwgPD0gMjAsc3VydmV5X3VuaXRdKQoKRmlzaEdsb2Iud2VsbHNhbXBsZWR5ZWFyc2NlbGxzX2NvbXBsZXRlLmZpbmFsIDwtIEZpc2hHbG9iLndlbGxzYW1wbGVkeWVhcnNjZWxsc19jb21wbGV0ZS4xMHllYXJbc3VydmV5X3VuaXQgJWluJSBzdXJ2ZXlfdW5pdHNfa2VlcCxdCgpzYXZlUkRTKEZpc2hHbG9iLndlbGxzYW1wbGVkeWVhcnNjZWxsc19jb21wbGV0ZS5maW5hbCwgaGVyZTo6aGVyZSgiZGF0YSIsImNsZWFuZWQiLCJGaXNoR2xvYi53ZWxsc2FtcGxlZHllYXJzY2VsbHNfY29tcGxldGUuZmluYWwucmRzIikpCgoKYGBg